{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "167e1376",
   "metadata": {},
   "source": [
    "# RedFin Summarizer\n",
    "This is a small project that uses a RedFin Scraper to gather information and links for houses on sale, <br>\n",
    "then uses GPT-5-nano or Llama 3.2 to generate a summary of the listing in place them into a neat table.\n",
    "\n",
    ">**WARNING**: If using GPT model, there is a risk of high token usage."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa80325a",
   "metadata": {},
   "source": [
    "### Redfin Scraper and Relevant Link Finder Implementation\n",
    "This portion of the code will focus on implementing the scraper for RedFin, which will gather the <br>\n",
    "listing links from a city-wide search page and return the contents of each listing url link into a combined output.\n",
    "\n",
    "The list of url links will be scrubbed for only relevant links using a similar prompt from day5 lesson."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fc1ae19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# redfin_scraper.py\n",
    "\n",
    "# Scraper Imports\n",
    "import json\n",
    "import requests\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from dotenv import load_dotenv\n",
    "from IPython.display import Markdown, display, update_display\n",
    "from openai import OpenAI\n",
    "from urllib.parse import urljoin, urlparse\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ddb19f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# API Key Check\n",
    "load_dotenv(override=True)\n",
    "api_key = os.getenv('OPENAI_API_KEY')\n",
    "\n",
    "if api_key and api_key.startswith('sk-proj-') and len(api_key)>10:\n",
    "    print(\"API Key is Correct!\")\n",
    "else:\n",
    "    print(\"Api Key Error.\")\n",
    "\n",
    "# Model Constants\n",
    "MODEL_GPT = 'gpt-4o-mini'\n",
    "MODEL_LLAMA = 'llama3.2'\n",
    "\n",
    "# OpenAI Client Instantiation\n",
    "openai = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1872953c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Content Scraper w/o Text Limits and Updated Link Scraper\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n",
    "}\n",
    "\n",
    "BASE_URL = \"https://www.redfin.com\"\n",
    "\n",
    "def fetch_website_contents(url):\n",
    "    \"\"\"\n",
    "    Return the title and contents of the website at the given url.\n",
    "    \"\"\"\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    title = soup.title.string if soup.title else \"No title found\"\n",
    "    if soup.body:\n",
    "        for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
    "            irrelevant.decompose()\n",
    "        text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
    "    else:\n",
    "        text = \"\"\n",
    "    return (title + \"\\n\\n\" + text)\n",
    "\n",
    "\n",
    "def fetch_website_links(url, base=BASE_URL):\n",
    "    \"\"\"\n",
    "    Return absolute links from the website at the given URL.\n",
    "    Relative links are converted to absolute URLs using the base URL.\n",
    "    Invalid links like '#' or 'javascript:void(0)' are ignored.\n",
    "    \"\"\"\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    \n",
    "    raw_links = [link.get(\"href\") for link in soup.find_all(\"a\")]\n",
    "    \n",
    "    absolute_links = []\n",
    "    for link in raw_links:\n",
    "        if not link or link.startswith(\"#\") or link.startswith(\"javascript:\"):\n",
    "            continue\n",
    "        if link.startswith(\"http://\") or link.startswith(\"https://\"):\n",
    "            absolute_links.append(link)\n",
    "        elif link.startswith(\"/\"):\n",
    "            absolute_links.append(urljoin(base, link))\n",
    "    return absolute_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "349357d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Links from Website\n",
    "links = fetch_website_links(url)\n",
    "# links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5ffea49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Similar Link System Prompt (Updated for RedFin Website)\n",
    "link_system_prompt = \"\"\"\n",
    "You are provided with a list of links found on a RedFin city search webpage.\n",
    "You are able to decide which of the links are relevant links to a property listing that is for sale,\n",
    "such links usually include state, city, and address, usually in the form:\n",
    "'*STATE*/*City*/*Address*/home/*12345678*', but without the '*' characters.\n",
    "\n",
    "You should respond in JSON as in this example:\n",
    "\n",
    "{\n",
    "    \"links\": [\n",
    "        {\"type\": \"property link\", \"url\": \"https://full.url/goes/here/about\"},\n",
    "        {\"type\": \"property link\", \"url\": \"https://another.full.url/careers\"}\n",
    "    ]\n",
    "}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "029d5d94",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Function for User Prompt\n",
    "def get_links_user_prompt(url):\n",
    "    user_prompt = f\"\"\"\n",
    "Here is the list of links on the website {url} -\n",
    "Please decide which of these are relevant web links for a property listing, \n",
    "respond with the full https URL in JSON format.\n",
    "Do not include News, Terms of Service, Privacy, or email links.\n",
    "\n",
    "Links (some might be relative links):\n",
    "\n",
    "\"\"\"\n",
    "    links = fetch_website_links(url)\n",
    "    user_prompt += \"\\n\".join(links)\n",
    "    return user_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32578001",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check User Prompt with Provided URL\n",
    "print(get_links_user_prompt(\"https://www.redfin.com/city/26659/CT/West-Hartford\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6a0d175",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Function for Relevant Link Selection\n",
    "def select_relevant_links(url):\n",
    "    response = openai.chat.completions.create(\n",
    "        model=MODEL_GPT,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": link_system_prompt},\n",
    "            {\"role\": \"user\", \"content\": get_links_user_prompt(url)}\n",
    "        ],\n",
    "        response_format={\"type\": \"json_object\"}\n",
    "    )\n",
    "    result = response.choices[0].message.content\n",
    "    links = json.loads(result)\n",
    "    print(f\"[INFO] Found {len(links['links'])} relevant links\")\n",
    "\n",
    "    return links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8c4a729",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check If Appropriate Links were Chosen\n",
    "select_relevant_links(\"https://www.redfin.com/city/26659/CT/West-Hartford\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e11c8212",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Function for Relevant Link Selection\n",
    "def fetch_page_and_all_relevant_links(url):\n",
    "    contents = fetch_website_contents(url)\n",
    "    relevant_links = select_relevant_links(url)\n",
    "    result = f\"## Landing Page:\\n\\n{contents}\\n## Relevant Links:\\n\"\n",
    "    for link in relevant_links['links']:\n",
    "        result += f\"\\n\\n### Link: {link['type']}\\n\"\n",
    "        result += fetch_website_contents(link[\"url\"])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b36ae5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check If Output Provides Necessary Information with Relevant Links\n",
    "print(fetch_page_and_all_relevant_links(\"https://www.redfin.com/city/26659/CT/West-Hartford\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79fea3d7",
   "metadata": {},
   "source": [
    "### Analyze Property Data and Generate Report\n",
    "This portion of the code uses a secondary \"one-shot\" prompt to generate a detailed report on the <br>\n",
    "property listings found from the above link search.\n",
    "\n",
    "General information about the city and simple, readily available details on the properties are <br>\n",
    "neatly organized in an easy-to-read format with necessary details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d453443",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Similar Secondary System Prompt (Updated for RedFin Website)\n",
    "real_estate_system_prompt = \"\"\"\n",
    "You are a real estate analysis assistant that analyzes the contents of several relevant pages from a RedFin City-Wide search result webpage.\n",
    "You create a detailed report on the city and each property listing for prospective buyers. Neatly organize the information for each listing into a table\n",
    "and include information such as address, price, beds, baths, sqft, features, description, property type, school rating, walkinh/biking/hiking scores, etc. \n",
    "Keep the information relevant to what a buyer must know when purchasing a home. Provide a summary of the city specified and city market notes, \n",
    "with a small table showing the averages of prices, square footage, lot size, build year, bed/bath numbers, etc.\n",
    "If any of the necessary information is not available, keep the heading but fill it with a 'null' occupier. \n",
    "\n",
    "Ignore information about RedFin or related to RedFin. Simply focus on the property data and the city.\n",
    "\n",
    "Respond in markdown without code blocks.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3945f098",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_city_name(url):\n",
    "    \"\"\"\n",
    "    Extract the city name from a Redfin URL and convert hyphens to spaces.\n",
    "    Example:\n",
    "        'https://www.redfin.com/city/26659/CT/West-Hartford' -> 'West Hartford'\n",
    "        'https://www.redfin.com/city/12345/NY/Maine' -> 'Maine'\n",
    "    \"\"\"\n",
    "    path = urlparse(url).path  # '/city/26659/CT/West-Hartford'\n",
    "    city_part = path.strip(\"/\").split(\"/\")[-1]  # 'West-Hartford'\n",
    "    city_name = city_part.replace(\"-\", \" \")  # 'West Hartford'\n",
    "    return city_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e94e65d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if City Name is Extracted Correctly\n",
    "extract_city_name(\"https://www.redfin.com/city/26659/CT/West-Hartford\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95a26cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Function for Creating Secondary User Prompt\n",
    "def get_real_estate_user_prompt(url):\n",
    "    city_name = extract_city_name(url)\n",
    "    print(f\"[INFO] Looking for property information in {city_name}...\")\n",
    "    user_prompt = f\"\"\"\n",
    "You are looking at a RedFin search for city called: {city_name}\n",
    "Here are the property listings for sale on its landing page;\n",
    "use this information to build a detailed report of the properties \n",
    "in this city and also provide a short background of the city.\n",
    "in markdown without code blocks.\\n\\n\n",
    "\"\"\"\n",
    "    user_prompt += fetch_page_and_all_relevant_links(url)\n",
    "    user_prompt = user_prompt\n",
    "    return user_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35ae910e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check If Output Provides Necessary Information\n",
    "print(get_real_estate_user_prompt(\"https://www.redfin.com/city/26659/CT/West-Hartford\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d82cae18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reused Function for Creating Final Output\n",
    "def create_report(url):\n",
    "    stream = openai.chat.completions.create(\n",
    "        model=\"gpt-4.1-mini\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": real_estate_system_prompt},\n",
    "            {\"role\": \"user\", \"content\": get_real_estate_user_prompt(url)}\n",
    "        ],\n",
    "        stream=True\n",
    "    )\n",
    "    response = \"\"\n",
    "    display_handle = display(Markdown(\"\"), display_id=True)\n",
    "    for chunk in stream:\n",
    "        response += chunk.choices[0].delta.content or ''\n",
    "        update_display(Markdown(response), display_id=display_handle.display_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "122a4b7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_report(url=\"https://www.redfin.com/city/26659/CT/West-Hartford\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm-engineering",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
